{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Reinforcement Learning <em> in Action </em>\n",
    "## N-Armed Bandits\n",
    "### Chapter 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch as th\n",
    "from torch.autograd import Variable\n",
    "from matplotlib import pyplot as plt\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This defines the main contextual bandit class we'll be using as our environment/simulator to train a neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContextBandit:\n",
    "    def __init__(self, arms=10):\n",
    "        self.arms = arms\n",
    "        self.init_distribution(arms)\n",
    "        self.update_state()\n",
    "        \n",
    "    def init_distribution(self, arms):\n",
    "        # Num states = Num Arms to keep things simple\n",
    "        self.bandit_matrix = np.random.rand(arms,arms)\n",
    "        #each row represents a state, each column an arm\n",
    "        \n",
    "    def reward(self, prob):\n",
    "        reward = 0\n",
    "        for i in range(self.arms):\n",
    "            if random.random() < prob:\n",
    "                reward += 1\n",
    "        return reward\n",
    "        \n",
    "    def get_state(self):\n",
    "        return self.state\n",
    "    \n",
    "    def update_state(self):\n",
    "        self.state = np.random.randint(0,self.arms)\n",
    "        \n",
    "    def get_reward(self,arm):\n",
    "        return self.reward(self.bandit_matrix[self.get_state()][arm])\n",
    "        \n",
    "    def choose_arm(self, arm):\n",
    "        reward = self.get_reward(arm)\n",
    "        self.update_state()\n",
    "        return reward\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we define our simple neural network model using PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(av, tau=1.12):\n",
    "    n = len(av)\n",
    "    probs = np.zeros(n)\n",
    "    for i in range(n):\n",
    "        softm = ( np.exp(av[i] / tau) / np.sum( np.exp(av[:] / tau) ) )\n",
    "        probs[i] = softm\n",
    "    return probs\n",
    "\n",
    "def one_hot(N, pos, val=1):\n",
    "    one_hot_vec = np.zeros(N)\n",
    "    one_hot_vec[pos] = val\n",
    "    return one_hot_vec\n",
    "\n",
    "arms = 10\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 1, arms, 100, arms\n",
    "\n",
    "model = th.nn.Sequential(\n",
    "    th.nn.Linear(D_in, H),\n",
    "    th.nn.ReLU(),\n",
    "    th.nn.Linear(H, D_out),\n",
    "    th.nn.ReLU(),\n",
    ")\n",
    "\n",
    "loss_fn = th.nn.MSELoss(size_average=False)\n",
    "\n",
    "env = ContextBandit(arms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we define the training function, which accepts an instantiated ContextBandit object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(env):\n",
    "    epochs = 5000\n",
    "    #one-hot encode current state\n",
    "    cur_state = Variable(th.Tensor(one_hot(arms,env.get_state())))\n",
    "    reward_hist = np.zeros(50)\n",
    "    reward_hist[:] = 5\n",
    "    runningMean = np.average(reward_hist)\n",
    "    learning_rate = 1e-2\n",
    "    optimizer = th.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    plt.xlabel(\"Plays\")\n",
    "    plt.ylabel(\"Mean Reward\")\n",
    "    for i in range(epochs):\n",
    "        y_pred = model(cur_state) #produce reward predictions\n",
    "        av_softmax = softmax(y_pred.data.numpy(), tau=2.0) #turn reward distribution into probability distribution\n",
    "        av_softmax /= av_softmax.sum() #make sure total prob adds to 1\n",
    "        choice = np.random.choice(arms, p=av_softmax) #sample an action\n",
    "        cur_reward = env.choose_arm(choice)\n",
    "        one_hot_reward = y_pred.data.numpy().copy()\n",
    "        one_hot_reward[choice] = cur_reward\n",
    "        reward = Variable(th.Tensor(one_hot_reward))\n",
    "        loss = loss_fn(y_pred, reward)\n",
    "        if i % 50 == 0:\n",
    "            runningMean = np.average(reward_hist)\n",
    "            reward_hist[:] = 0\n",
    "            plt.scatter(i, runningMean)\n",
    "        reward_hist[i % 50] = cur_reward\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Backward pass: compute gradient of the loss with respect to model\n",
    "        # parameters\n",
    "        loss.backward()\n",
    "\n",
    "        # Calling the step function on an Optimizer makes an update to its\n",
    "        # parameters\n",
    "        optimizer.step()\n",
    "        cur_state = Variable(th.Tensor(one_hot(arms,env.get_state())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt0lPWd+PH3JxeSgJiUWxNAhFCLWk2FTa2X0qrZQi1N\ncW3XQ9XVXvawdmtF+sMeqaeW2rV2V1vEtb96WHvzV1vXKopZrNhG2lJUWgQaoIBIRCUkcluCxCTk\n8vn98TwzzExmJs8k88z18zonJzPfeTLzfUKYzzzf7+f7+YqqYowxxgAUpLsDxhhjMocFBWOMMUEW\nFIwxxgRZUDDGGBNkQcEYY0yQBQVjjDFBFhSMMcYEWVAwxhgTZEHBGGNMUFG6O5CocePG6dSpU9Pd\nDWOMySqvvPLKYVUdP9hxWRcUpk6dyqZNm9LdDWOMySoi8oaX42z4yBhjTJAFBWOMMUEWFIwxxgRZ\nUDDGGBNkQcEYY0yQBQVjjDFBFhSMMcYEWVAwxhgTZEHBmBzX3tDAnivq2HnOuey5oo72hoZ0d8lk\nsKxb0WyM8a69oYHWb96JdnUB0HvgAK3fvBOA8vr6dHbNZCi7UjAmhx1cfn8wIARoVxcHl9+fph6Z\nTGdBwZgc1tvamlC7MRYUjMlhRVVVCbUbY0HBmBw2YfGtSGlpWJuUljJh8a1p6pHJdDbRbEwOC0wm\nH1x+P72trRRVVTFh8a02yWxisqBgTI4rr6+3IJAET29p4d61uzlwrJOJFWXcNncGV82clO5uJZ2v\nw0cislhEdojIdhH5lYiURjx+mYi0i8hW9+tOP/tjjDFD8fSWFpau2kbLsU4UaDnWydJV23h6S0u6\nu5Z0vgUFEZkE3ALUqup5QCGwIMqh61X1AvfrLr/6Y4wxQ3Xv2t109vSFtXX29HHv2t1p6pF//J5o\nLgLKRKQIGAkc8Pn1jDEm6Q4c60yoPZv5FhRUtQW4D3gTaAXaVfX5KIdeIiJNIvIbEflAtOcSkYUi\nsklENh06dMivLhtjTFQTK8oSas9mfg4fvQeYD0wDJgKjROT6iMM2A1NUtQb4T+DpaM+lqitVtVZV\na8ePH+9Xl02SPL2lhUu/9wLTbl/Dpd97ISfHXU1+uW3uDMqKC8PayooLuW3ujDT1yD9+Dh/9PfC6\nqh5S1R5gFXBJ6AGqelxVT7i3nwWKRWScj30yPsunCTkT3ZrmNcx5Yg41P69hzhNzWNO8Jt1dGrar\nZk7inqvPZ1JFGQJMqijjnqvPz8nsIz9TUt8ELhKRkUAnUAdsCj1ARCqBt1VVReRCnCB1xMc+GZ/F\nm5CL9R/o1Y1tvLR6LyeOdnPamBIunj+d93+4MhXdNUm2pnkNy15cRlefU2+ptaOVZS8uA2Be9bw0\n9mz4rpo5KSeDQCQ/5xQ2Ak/gDBFtc19rpYjcJCI3uYd9FtguIn8FHgAWqKr61Sfjv0Qn5F7d2Ma6\nR3dx4mg3ACeOdrPu0V28urHNtz4a/6zYvCIYEAK6+rpYsXlFmnpkEuXr4jVV/RbwrYjmh0IefxB4\n0M8+mNSaWFFGS5QAEGtC7qXVe+k92R/W1nuyn5dW77WrhWFY07yGFZtX0NbRRuWoShbNWpSST+pt\nHdGDeax2k3ms9pFJqkQn5AJXCF7bzeACQzitHa0oGhzCScXYfuWo6IE8VrvJPFbmwiRVYMzVazmA\n08aURA0Ap40pCbvf1NREY2Mj7e3tlJeXU1dXR01NTfJPIAfEG8Lx+2ph0axFYXMKAKWFpSyatcjX\n1zXJY0HBJF0iE3IXz5/Oukd3hQ0hFY0o4OL504P3m5qaaGhooKenB4D29nYa3C0lLTAMlM4hnEDQ\nScfQVUo1PQ6Nd0H7fiifDHV3Qs016e5VUlhQyCFPth3lnuZWWrp7mFRSzNLqKj5TOSbd3YorMG8Q\nL/uosbExGBACenp6aGxstKAQReWoSlo7Bm6ik6ohnHnV83IvCIRqehwaboEed+6s/S3nPuREYLCg\nkCOebDvKkt1v0dnvJG/t7+5hye63ALIiMMSbVG5vb0+oPR0yqYKmDeH4rPGuUwEhoKfTac+BoGAT\nzTninubWYEAI6OxX7mnO/m0Xy8vLE2pPtUxbsDeveh7LLllG1agqBKFqVBXLLlmW25/eU6l9f2Lt\nWcauFHJES3dPQu3ZpK6uLmxOAaC4uJi6uro09uqUoSzY81vOD+GkU/lkZ8goWnsOsKCQIyaVFLM/\nSgCYVFKcht4kV2DeIJB9dOaZbUydtpVDh3/Khg1VVE9fQlXl/LT1LxcqaHZsOcjxtfvoO9ZNYUUJ\np8+dyqiZE7w/QQ5PvA5Qd2f4nAJAcZnTngMsKOSIpdVVYXMKAGUFwtLq3NigvaamhpqaGlrbVrNr\n1x309zv/Ibu6D7Br1x0A4YFhGG9Sib5BJrpgL9N0bDnIsVV70B4nA6zvWDfHVu0B8BYYcnzidYDA\nOeVoELSgkCMCk8nZln2UqOa99wUDQkB/fyfNe+87FRQ8vklFq7k0aURBwm+Qt82dwdJV28KGkLKp\ngubxtfuC5xugPf0cX7vPW1BI4sRrrPUoGZdZV3NNzgSBSBYUcshnKsfkXBCI1NUdfeI8rN3Dm1Sg\n5lJgfUSg5tKVY0soSPANMtEFe5mm71j01eOx2gdI0sRrrPUojV39PNApWZlZl40sKOSx9oYGDi6/\nn97WVoqqqpiw+NaM3+C9tKSKru6BG/iVloQMk0W8Ga0ZNZIV76mgrQgqn5jDolmLOLy6ImrNJens\njfq6g71BZnMFzcKKkqjnV1hREuXoKJI08Rq5HmX8+GamTtvK1098iE4JD8iBzDoLCslnQSFPtTc0\n0PrNO9EuJ5e998ABWr/pTJRlcmConr4kbE4BoKCgjOrpS04dFPImtWbUSJaNG0NXgZN9HagD9Pmj\n3wNkwPN39ikjCwe2e36DzEKnz50aNmQGIMUFnD53qrcnGMLE687161j/2CO8c+Qwo8eOY/aCG8LW\nnYwf38xZ73+ZwsI+jhB9i5WW7p6Mn+DOxrLwtk4hTx1cfn8wIARoVxcHl9/v+Tla21azYcNsGl94\nHxs2zKa1bXWyuzlAVeV8zj77bkpLJgJCaclEzj777vBJ5ro7nTclYMV7KoIBIaCrr4t3S49Hff7m\nQkGKw49P6A0yC42aOYGKq88KBr7CihIqrj7Le/ZRzTVQ/wCUnwGI873+gZhvzjvXr+P5lQ/yzuFD\noMo7hw/x/MoHGVl6KvBOnbaVwkJnjmYch6M+zxePvOAEo/a3AD01d9T0uOdz91O2loW3K4U81dsa\nfWw+Vnskz1lAPqiqnB//NUKyQ9pi/IW/OHk1n3jz8wNqLlX/w1lUjCgYXnpmFho1c8LwzjGBidf1\njz1C78nw4arek92MOLifnrET6enpoaSkI/jYNTzKw/plTkppsK2sQPjGvoeHNcEdmmXWfNphNhXv\n5XjniaQVXMzWsvAWFPJUUVUVvQcGjs0XVXlLYfWUBZRO7ptU5RNzwuoAXbqjj2t/r4w7vpFDM0bz\nevV8OjoLBlza53oQSKd3jkT/5N934E3qP/8vNDY20t09itJSJzBcyp8AeFyv44iMZ1LJCJZWVzFq\n3cC/X8DTBHdoGu5rBa2s79lFX6/zBh5acHHPhMlDznrK1rLwvg4fichiEdkhIttF5FciIaHeeVxE\n5AEReU1EmkRklp/9MadMWHwrUhr2z4GUljJh8a2eft5TFlAGWDRrEaWFznleuqOPf3lWGX/cmU2Y\nsPt3XLTuNq6f18GN3700oz+95ZLRY6PPEYweO46amhoWL17MrFnfoaDg1DqPS/kT/1nwNTaf8yab\nLvmA88YcayLbwwR3aBrupqJm+iT8E31PTw8PvLKNJbvfYn93D8qprKcn2456Os/I8u+DtWcK34KC\niEwCbgFqVfU8oBBYEHHYlcBZ7tdC4Ed+9ceEK6+vp+o7d1E0cSKIUDRxIlXfuWvQSeampiaWL19O\nV9fIqI+HZQFlgNA6QNf+XimNSC5KdB7FDN/sBTdQNCL8jbFoRAmzF9wQvJ/o3FGQx5XFodlWJ6Qr\n6jG/r5w2rHpiF8+fTtGI8LfYyLLwmcjv4aMioExEeoCRQOT13nzgEXdf5pdFpEJEqlQ1sz5u5qjy\n+vqEMo1C88j3vX5BMDskYEAWUIYI1AHa+c1zgYFbgHudRzHJcc7sywEGZB8F2gMSmTtKNPsoNA33\nNC2NGhhOlERfke61npiXsvCQeRtI+RYUVLVFRO4D3gQ6gedV9fmIwyYBoQnO+902+1+agULzyA8d\nqgacLJGSkg7KSicmVIMoHWskhjuPkgrDrkHko2SuKj5n9uUDgsCQDHFlcWgabm1vNeuLd4UNIRUX\nFzOhUDjYP/BnE6knNlhZ+EzcQMq3oCAi78G5EpgGHAN+LSLXq+ovhvBcC3GGl5gyZUpS+5mN0vXJ\nInL/gkOHqoPBYdmyZd6fJ01rJCYsvjXsdSGxeRS/DbsGkY+yeb+OaAK/z+Nr9/G+Y1UUFBcPyD46\ne8Jk3+uJZeIGUn4OH/098LqqHgIQkVXAJUBoUGgBzgi5P9ltC6OqK4GVALW1tQOv//NIOj9ZlJeX\nR93YJtF9DeKtkfAzKASeO1NXcQ+7BlGSRPvQcc+Jwpjj69kYFCA8DXcy8FG3vbVtNc17v0LF4VYW\nFn2Kx4uup623yJeaS5m4gZSfQeFN4CIRGYkzfFQHbIo45hngZhF5DPgw0G7zCfH59cnCy8rLZO1r\nMNw1EsOR6DyKX6L9vkcOtwZREsT60NFySfS9GXJhv45QketvLuxt4KKC33H2OXf7kmqdrA9ayeRb\n9pGqbgSeADYD29zXWikiN4nITe5hzwLNwGvAfwH/6ld/coUfnyy8rrysqamhvr4++AdbXl5OfX19\nwsEo1hh+Jo3t+ynW77u/LPpnNM8lNpoeh+XnwbIK5/sQVvbG+tAx+mT0vSGybb+OnevXsfIrX+D7\nC+pZ+ZUvsHP9urDH462/8UNdXR3FxeG/w8AHrSfbjlL74g6q1m2l9sUdnlNhh8vX7CNV/RbwrYjm\nh0IeV+ArfvYh10R+sggUDSsp6WDDhsYhbTiTyMrLwL4Gw5HpY/t+i/X7/ltnL+cXFwytBlGS9jSI\n9eHiQ69t58XzLszo/TrWNK9hxeYVtHW0UTmqkkWzFoXtPhcorxFYTR0orwGnMqJSvf4mcgOpwHDd\nnoj5jFTO4VjtowzQ3tDAnivq2HnOuey5oo52d44gmtBPFoGiYaWlHYicKjWRaA2iVK+8HOoaiVwR\n6/e691jP0GsQxSsXnoBYwxa1J09w34wzmFxSjACTS4q5b8YZGTOfsKZ5DcteXEZrRyuKBgsfrmle\nEzwmVnmN9Y89Erwfa52Nn+tvAgv2li1bxuLFi6mpqUnrnutW5iLNEs3ECf1kEVo0LGAopSZOG1MS\n9Y3Kz5WXmTK2nw7xft9DrkGUpD0N4s0b1WTwfh0rNq+gqy88eaGrr4sVm1cErxZildcIbfdUhTcF\n0rnnul0ppNlQqpUGPlmUlr4b9fFEL3WzdeVltvLl9z2Mkg+hkjVvlGptHdErj4a2xyuvEeBpJXUK\nxJqrScUcjl0ppFm8TJxoNedDF/zE23Dm6S0tnncC87ryMhNl3DaNHvjy+07iZvLJmDcK8LJIMRkL\n9ipHVYYVPgxtD5i94IawOQUYWF4DPKykToF07rkuzlxv9qitrdVNmyIzW7PXnivqoq6ybZt2Bk3j\nRg/4A56z8OZgYIhMnwPnUvcNvsu/N44YsGfwPVefn7W7g0UTuaAKnP84mTTW7cVgwd8zDxvOpHLT\nl8ihUXASCkLnjyIX7IEzuZ7Qfg6cmlMIHUIqLSxl2SXLBkw2J+V3nQLJ/sAjIq+oau2gx1lQSK9Y\n/3H+MGsGJzpODDh+9LjxLPzhT4P3nYU299HV3UppSRXV05fw2Z+NpuXYwBTCSRVlbLj9Cn9OJA1q\nX9zB/ihjrJNLitl0yQfS0KPERWbEwMDgnyyR+1I7r1XA5ded7UtgiPWBp2jiRM56oRGA1u/9OeZW\noFW3X5jQ6w2WfZRJ0lHOxGtQsOGjNIu1yvbEoyujHh85WRbtUvfAsTVEcyBKoMhm6ZyMS5Z4GTHJ\nDgrJ3vRlsCFKL4sUYy3MG8qCvUDhw0yXyeVMwIJCRoiWiTP6uVXOdoURYk2WhZpYURb1SmFiRfSq\nj9lqUklx1CuFbFpQ5SUjJlmSmXr89JYWlq7aFhyibDnWydJV2wCCgSGyAGHbhFr2Vn+a7pIx/Pkb\nG7h4/nRGh1QrDZXLe2LHK2dyvOqlAVf+qZ7fsOyjDOWl5nwst82dQVlxYVhbWXEht82dwZrmNcx5\nYg41P69hzhNzwvK4s83S6irKCiSsLdMWVA0mXkZMYO+KZcuWsXz5cpqamob1Wsnc9OXetbvD5qwA\nOnv6uHft7uD90I2c2ibUsmvGtXSXjgWR4Crud99XkXd7Yse6Cjpauo5du+5wk0d0yOuOhsuCQoY6\nZ/blzFl4M6PHjQcRRo8b73mc+aqZk7jn6vOZVFGG4Mwl3HP1+RSXbx10gY+fkh2QPlM5JmkLqgYr\nf5CQBMpNxAr+Uz42l4aGhuAK4xEjtrB///U0vvA+NmyYPaQ3imSmwsYaigxtD12kuLf60/QXhp9n\n78l+/rT50NAX7GWQRP62Y10FHZ6xKqUlNmKxieY8Midiv+KAqlFVPP/ZyK0ukstrdkg6JDLZG21i\nP+zyPrLcBDipofUPxCw3ES0j5rlNW4MBIbByPXJDo6Hkzycr++jS772QUDLDD296IeZzfeWh7E5+\nSPRvO1bG1a7LbyTaJlAg1F3x2rD7aRPNOSYZ2QpeFvgEJDt1z8uK03TxOtkbmQIcuLwHTr05xys3\nESMoRNtw5r8b/xC8HW/l+sbWWs/rUWDwTV+8um3ujLA5BTg1RBlNOlbNp0rk3/alO/q49vcnGPdv\nS9gz8QcD1maE7uUQ+v+59N3Y645SyYJCFkhWtoKXBT7grXBYohIJSAGpStvzOtkbWUFzAx/h8f7r\nOLJzPJOadzh55EkqNxFa+LCkpCPqMeten8iju+NP9vol8PxeA9LF86dHTYfNhVXzoX/Dl+7o41+e\nPbUXeKyyNdHKmVS3ZUaJDZtTyALxshUSsWjWIkoLS8PaSgtLWTRrUVibl8JhiYoMPLHaA2P7v/7S\n1zn02PbgpFwgEHZsOTjkPsTipfwBhJcP2cBHeJgvc1gmoEiwimXHaROjv0iC5SZCCx92d4+KeszT\ne+cPOtnrp6tmTmLD7Vfw+vfmseH2Kwa9Qrn8urODVwanjSnxbX1EqoX+DV/7+1MBIWCwsjUBmVJi\nw64UskCycrkDwzSDLfDxI01y0axFUcddQwNS6BXKxyZ/hiIJTy31axcyr+UPQsuKPM51nJTwANvZ\nr3x36j9z9+7/GHa5idDCh/tev4D3z9hIQcGpd5uCgjKOdJ4e9WczdT1KsoauMk3o3/bY49GP8bqB\nVCaU2LCgkAUKk5jL7WWBz+ix44a8RiLe60L8gBR6hTKyKPobnh+7kAWGxAabQwmtoHmY6L+Ln4y9\ngrvrJw9absKL0BpE0Sa4J748cljrUbKp5EOivNRcSpbQv+0jp7/F+CiBoaiqKmt+375lH4nIDOC/\nQ5qqgTtV9f6QYy4DVgOvu02rVDVuAfh8zD5KVn0YryLnFKaMOoeaMZcxsmg0RRWliY3te6jHE/D9\nBfXg/j1+avJNjCoeWNs/XvmDRIoAJiJ0z+Izz2xj6rStfLX/Lg7LwN9BZImN0NIL8/aM5nN/7Kf4\nUHtS3qgiF5CB9xpXqSyvkWpeai6l+rU7vvhP/OkvG9L6+0579pGq7gYucDtTCLQAT0U5dL2qfsqv\nfmSbeCmDqaqVEvrJ+T1d4/nQuE8Eh3ISmuROcDew0CuUpv/9Ax8adyVFBaeGkOItavKywnYoIvcs\nfuONSg4cqOdLH5/AA50St4plaKripTv6uObZoxQPMgGZiEQne0OlsrxGqsUrR+93UIhVtua/n1uV\nNb/vVA0f1QF7VfWNFL1eVoosWBZY9QnOeGwqF/QE0iSjFSzzPLafYHpm6Nj+mx07ATxfocRbYTuc\noBBrz2I2rOO+z90Yt4plaKpivAnI4bxRXTVz0pDOL5XlNVLNS80lP0UrW/OOx1pmmSBVQWEB8KsY\nj10iIk04VxJLVHVHivqUcZJdsCwZhjXJnWB6ZuTY/v+WHaZ//kjOmP3RQV/KywrboYi1Z3F7ezuf\nGWQnstBUxeFOQCabH/NGmcJLzaVU/3/Kpt+37ympIjIC+DTw6ygPbwamqGoN8J/A0zGeY6GIbBKR\nTYcODfzF5opU75XsRazJbE+T3EPYDeyc2Zez8Ic/5f881sDCH/7U86V1rMnV4RYBjLVncaz2UKGp\nikeiz5tTVJWeOk3Dqa2V6bzUXHp1Y+z1MX7Ipt93KtYpXAlsVtW3Ix9Q1eOqesK9/SxQLCIDQqeq\nrlTVWlWtHT9+vP89TpNkFixLltPnTh16wbK6O510zFBD3A1sMPGKAA5H6HqBgMCexYMJXRfyy8uE\nrojrciktZcLiWxPu05NtR6l9cQdV67ZS++IOnmw7mvBzDKe2VqbzUnPppdV7U9qnbPp9+177SEQe\nA9aq6k+jPFYJvK2qKiIXAk8AZ2qcTuVy9lGqN0HxalgrixPIPhquVGQflZeXO5vYe9yuMtnZR7my\n29xwJPLvnMs1lxKVETuvicgo4E2gWlXb3babAFT1IRG5Gfgy0At0Al9T1RfjPWcuBwVI7XaJJvvk\nwm5zw5FoGu7Pv7EhZs2lG797qa99zTTDTkkVkVnxflBVNw/25KraAYyNaHso5PaDwIODPU8+ydVV\nnyY5cmG3ueFINMssl2su+SVe9tH33e+lQC3wV0CAGmATcLG/XTPGRMqF3eaGI9Ess8AHLLv69i5m\nUFDVywFEZBUwS1W3uffPA5alpHc5LJs2GTeZY2l1VdQ5hWzabW44hrLVrF19J8ZL9tGMQEAAUNXt\nwDn+dSn3BVa6pmsHNJO9krnbXDbyK8vMnOJl8do2EXkY+IV7/zpgeJvF5pFomRL/d2/mbjhjMt9g\ni+Zy2XBKexhvBs0+EpFSnAyhwLLSPwI/UtWu2D/ln2zKPoqVKVH0vq8Tbds9QWi60Yd4m8K0UGNM\nZkpKQTy3kN2PVfU6YHmyOpcvYmVKnN5bgRb974DjY21EMywJFqUzxuS3uHMKqtoHnOmWqjAJipUR\n0fn2HE87oCVFvKJ0xhgTwcucQjOwQUSeAYKbxarqD3zrVY6IlSkxoeASvnHJzNRkHyVpz2BjTH7w\nEhT2ul8FwGh/u5Nbbps7I+qcwm1zZzCvelJqJpXLJztDRtHajTEmwqBBQVW/nYqO5KKMyJSouzN8\nTgF8K0pnjMl+gwYFERkPfB34AM7qZgBUNb+qSQ3RUDdBSZrAZLJlHxljPPAyfPQozl7LnwJuAm4E\ncndTg1xUc40FAWOMJ15WNI9V1R8DPar6B1X9ImBXCSZMMmr8G2PSz8uVQqD6VquIzAMOAPm5nNJE\nFVnjf393D0t2O5Pb+bry1phs5eVK4d9EpBz4P8AS4GFgsa+9MlnlnubWsAJtAJ39yj3N6dl/2Bgz\ndF6uFH7nlrRoBzJv7ziTdvle49+YXOIlKGwXkbeB9e7XnwK7qBkDVuPfmFwy6PCRqr4P+BywDZgH\n/FVEtg72cyIyQ0S2hnwdF5FbI44REXlARF4TkabBdnszmWlpdRVlBRLWlk81/o3JJV7WKUwGLgVm\nAx8EdgB/GuznVHU3cIH7HIVAC/BUxGFXAme5Xx8GfuR+N1kkMJl8T3MrLd09TCopZml1lU0yG5OF\nvAwfvQn8Bfiuqt40xNepA/aq6hsR7fOBR9Sp3/2yiFSISJWq2gxllsnnGv/G5BIvQWEm8BHgWhG5\nHdgD/MFdu+DVAuBXUdonAaGFefa7bWFBQUQWAgsBpkyZksDLpk5r22qa995HV3crpSVVVE9fQlXl\n/HR3yxhjEuKl9tFfRSRQFG82cD3wMcBTUHDLbn8aWDrUTqrqSmAlOJvsDPV5kq2pqYnGxkZGjNjC\n+2dspKCgF4Cu7gPs2nUHgAUGY0xWGXSiWUQ2AS8B/wDsBD6qqmcm8BpXAptV9e0oj7UAZ4Tcn+y2\nZbympiYaGhpob29n6rStwYAQ0N/fSfPe+9LUO2OMGRovw0dXqupwah19juhDRwDPADeLyGM4E8zt\n2TKf0NjYSE+Pk4ZZUtIR9Ziu7qw4FWOMCfKyorlARH4sIr8BEJFzReRLXp5cREYBHwdWhbTdJCKB\nCetncTbxeQ34L+BfE+l8OrW3n1qq0d09KuoxpSWWkmmMyS5egsLPgLXARPf+q8CtMY8Ooaodqjo2\ndLGbqj6kqg+5t1VVv6Kq01X1fFXdlFj306e8vDx4e9/rF9DXVxj2eEFBGdXTl6S6W8YYMyxegsI4\nVX0c6AdQ1V6gL/6P5L66ujqKi50Vu4cOVbPn1Yvo7hoFCKUlEzn77LttktkYk3W8zCl0iMhYQAFE\n5CKcOkh5raamBnDmFtrb2zl5ciaTJy8JthtjTDbyEhS+hjMhPF1ENgDjgX/0tVdZoqamxoKAyWm2\n/ib/eFmnsFlEPgbMAATYrapW/tKYHNfatppdu+6gv9/Z39vW3+QHL3MKqGqvqu5Q1e3AZSLyW5/7\nlZuaHofl58GyCud70+Pp7pExMTXvvS8YEAJs/U3uixkUROQKEXlVRE6IyC9E5Hx3Idv3cArXmUQ0\nPQ4Nt0D7W4A63xtuscBgMlasdTa2/ia3xbtS+D5OvaGxwBM4q5p/pqp/p6qr4vyciabxLugJ/9RF\nT6fTbkwGirXOxtbf5LZ4QUFV9feq2q2qTwMtqvpgqjqWc9r3J9ZuTJpVT19CQUFZWJutv8l98Saa\nK0Tk6tBjQ+/b1UKCyie7Q0dR2o3JQIHJZMs+yi/xgsIfgPqQ+38Mua+ElK4wHtTd6cwhhA4hFZc5\n7cZkqKrK+RYE8kzMoKCqX0hlR3JezTXO98a7nCGj8slOQAi0G2NMBvCyeM0kS801FgSMMRnN0zoF\nY4wx+cHlDTYcAAAQl0lEQVSCgjHGmCBPw0cicgkwNfR4VX3Epz4ZY4xJk0GDgoj8P2A6sJVTJbMV\nsKBgjDE5xsuVQi1wrqpqok8uIhXAw8B5OIHki6r6UsjjlwGrgdfdplWqakt8jTEmTbwEhe1AJTCU\ngicrgOdU9bMiMgIYGeWY9ar6qSE8tzHGmCTzEhTGAX8TkT8D3YFGVf10vB8SkXLgo8Dn3eNPAieH\n3FNjjDG+8xIUlg3xuacBh4CfisgHgVeARaraEXHcJSLSBLQAS1R1xxBfzxhjzDB52WTnD8N47lnA\nV1V1o4isAG4HvhlyzGZgiqqeEJFPAk8DZ0U+kYgsxKnYypQpU4bYHWOMMYMZdJ2CiFwkIn9x91U4\nKSJ9InLcw3PvB/ar6kb3/hM4QSJIVY+r6gn39rNAsYiMi3wiVV2pqrWqWjt+/HgPL22MMWYovCxe\nexD4HLAHKAP+GfjhYD+kqm3AWyIyw22qA/4WeoyIVIqIuLcvdPtzxHPvjTHGJJWnxWuq+pqIFKpq\nH84cwRZgqYcf/SrwqJt51Ax8QURucp/zIeCzwJdFpBfoBBYMJfXVGGNMcngJCu+6b+pbReQ/cFJT\nve7tvBVnnUOoh0IefxDnSiTjvLqxjZdW7+XE0W5OG1PCxfOn8/4PV6a7W8YY4ysvb+7/5B53M9AB\nnAF8xs9OpdurG9tY9+guThx1MnBPHO1m3aO7eHVjW5p7Zowx/vKSffSGiJQBVar67RT0Ke1eWr2X\n3pP9YW29J/t5afVeu1owxuQ0L9lH9Th1j55z718gIs/43bF0ClwheG03xphc4WX4aBlwIXAMgvME\n03zsU9qdNqYkoXZjjMkVXoJCj6q2R7TldIbQxfOnUzQi/FdTNKKAi+dPT1OPjDEmNbxkH+0QkWuB\nQhE5C7gFeNHfbqVXYN7Aso+MMfnGS1D4KnAHTjG8XwFrge/42alM8P4PV1oQMMbkHS/ZR+/iBIU7\n/O+OMcaYdIoZFAbLMBqsdLYxxpjsE+9K4WLgLZwho42ApKRHxhhj0iZeUKgEPo5TDO9aYA3wK9vv\nwBhjclfMoOAWv3sOeE5ESnCCw+9F5NtuzaK8sXP9OtY/9gjvHDnM6LHjmL3gBs6ZfXm6u2WMMUkX\nd6LZDQbzcALCVOAB4Cn/u5U5dq5fx/MrH6T3pLOa+Z3Dh3h+pRMTLTAYY3JNzMVrIvII8BLOxjjf\nVtUPqep3VLUlZb3LAOsfeyQYEAJ6T3az/rFH0tQjY4zxT7wrhetxqqIuAm5x98IBZ8JZVfV0n/uW\ncu0NDRxcfj+9ra0UVVUxYfGtvHPkcNRjY7UbY0w2izen4GnPhFzR3tBA6zfvRLu6AOg9cIDWb97J\nabNmcKLjxIDjR48dsGuoMcZkPV/f+EWkQkSeEJFdIrJTRC6OeFxE5AEReU1EmkRkVqzn8tvB5fcH\nA0KAdnXx/tajFI0IL4RXNKKE2QtuSGX3jDEmJfy+GlgBPKeqZwMfBHZGPH4lcJb7tRD4kc/9iam3\ntTVqe+W+/cxZeDOjx40HEUaPG8+chTfbJLMxJid52qN5KESkHPgo8HkAVT0JnIw4bD7wiLsv88vu\nlUWVqkZ/h/ZRUVUVvQcORG0/a/blFgSMMXnBzyuFacAh4KciskVEHhaRURHHTMJZNR2w321LuQmL\nb0VKS8PapLSUCYtvTUd3jDEmLfwMCkU46aw/UtWZOJlMtw/liURkoYhsEpFNhw4dSmYfg8rr66n6\nzl0UTZwIIhRNnEjVd+6ivL7el9czxphM5NvwEc6n/v2qutG9/wQDg0ILcEbI/cluWxhVXQmsBKit\nrfVtg5/y+noLAsaYvObblYKqtgFvicgMt6kO+FvEYc8AN7hZSBcB7emYTzDGGOPw80oBnA16HhWR\nEUAz8AURuQlAVR8CngU+CbwGvAt8wef+GGOMicPXoKCqW4HaiOaHQh5X4Ct+9sEYY4x3ebVq2Rhj\nTHx+Dx/lpSfbjnJPcyst3T1MKilmaXUVn6kck+5uGWPMoCwoJNmTbUdZsvstOvudJKn93T0s2e0s\nxbDAYIzJdDZ8lGT3NLcGA0JAZ79yT7MlVRljMp8FhSRr6e5JqN0YYzKJBYUkm1RSnFC7McZkEgsK\nSba0uoqyAglrKysQllZXpalHxhjjnU00J6hjy0GOr91H37FuCitKOH3uVEbNnBB8PDCZbNlHxphs\nZEEhAR1bDnJs1R60px+AvmPdHFu1B2BAYLAgYIzJRjZ8lIDja/cFA0KA9vRzfO2+9HTIGGOSzIJC\nAvqOdSfUbowx2caCQgIKK0oSajfGmGxjQSEBp8+dihSH/8qkuIDT505NT4eMMSbJbKI5AYHJ5HjZ\nR8YYk80sKCRo1MwJFgSMMTnLho+MMcYEWVAwxhgT5OvwkYjsA94B+oBeVa2NePwyYDXwutu0SlXv\n8rNPxhhjYkvFnMLlqno4zuPrVfVTKeiHMcaYQdjwkTHGmCC/g4ICvxORV0RkYYxjLhGRJhH5jYh8\nINoBIrJQRDaJyKZDhw7511tjjMlzfg8ffURVW0RkAvBbEdmlqn8MeXwzMEVVT4jIJ4GngbMin0RV\nVwIrAWprazXycWOMMcnh65WCqra43w8CTwEXRjx+XFVPuLefBYpFZJyffTLGGBObb0FBREaJyOjA\nbWAOsD3imEoREff2hW5/jvjVJ2OMMfH5OXz0XuAp9z2/CPilqj4nIjcBqOpDwGeBL4tIL9AJLFBV\nGx4yxpg08S0oqGoz8MEo7Q+F3H4QeNCvPkTV9Dg03gXt+6F8MtTdCTXXpLQLxhiTqfKr9lHT49Bw\nC/R0Ovfb33LugwUGY4wh39YpNN51KiAE9HQ67cYYY/IsKLTvT6zdGGPyTH4FhfLJibUbY0yeya+g\nUHcnFJeFtxWXOe3GGGPyLCjUXAP1D0D5GYA43+sfsElmY4xx5Vf2ETgBwIKAMcZElV9XCsYYY+Ky\noGCMMSbIgoIxxpggCwrGGGOCLCgYY4wJsqBgjDEmyIKCMcaYIAsKxhhjgiwoGGOMCfI1KIjIPhHZ\nJiJbRWRTlMdFRB4QkddEpElEZvnZH2OMMfGloszF5ap6OMZjVwJnuV8fBn7kfk+qp7e0cO/a3Rw4\n1snEijJumzuDq2ZOSvbLGGNM1kt37aP5wCPuvswvi0iFiFSpamuyXuDpLS0sXbWNzp4+AFqOdbJ0\n1TYACwzGGBPB7zkFBX4nIq+IyMIoj08C3gq5v99tS5p71+4OBoSAzp4+7l27O5kvY4wxOcHvK4WP\nqGqLiEwAfisiu1T1j4k+iRtQFgJMmTIloZ89cKwzoXZjjMlnvl4pqGqL+/0g8BRwYcQhLcAZIfcn\nu22Rz7NSVWtVtXb8+PEJ9WFiRVlC7cYYk898CwoiMkpERgduA3OA7RGHPQPc4GYhXQS0J3M+AeC2\nuTMoKy4MaysrLuS2uTOS+TLGGJMT/Bw+ei/wlIgEXueXqvqciNwEoKoPAc8CnwReA94FvpDsTgQm\nky37yBhjBidO4k/2qK2t1U2bBix5MMYYE4eIvKKqtYMdZyuajTHGBOV9UFjTvIY5T8yh5uc1zHli\nDmua16S7S8YYkzbpXryWVmua17DsxWV09XUB0NrRyrIXlwEwr3peGntmjDHpkddXCis2rwgGhICu\nvi5WbF6Rph4ZY0x65XVQaOtoS6jdGGNyXV4HhcpRlQm1G2NMrsvroLBo1iJKC0vD2koLS1k0a1Ga\nemSMMemV1xPNgcnkFZtX0NbRRuWoShbNWmSTzMaYvJXXQQGcwGBBwBhjHHk9fGSMMSacBQVjjDFB\nFhSMMcYEWVAwxhgTZEHBGGNMUNaVzhaRQ8AbQ/zxccDhJHYnG9g55wc75/wwnHM+U1UH3boy64LC\ncIjIJi/1xHOJnXN+sHPOD6k4Zxs+MsYYE2RBwRhjTFC+BYWV6e5AGtg55wc75/zg+znn1ZyCMcaY\n+PLtSsEYY0wceRMUROQTIrJbRF4TkdvT3Z/hEJGfiMhBEdke0jZGRH4rInvc7+8JeWype967RWRu\nSPvficg297EHRERSfS5eiMgZIrJORP4mIjtEZJHbnsvnXCoifxaRv7rn/G23PWfPOUBECkVki4j8\nj3s/p89ZRPa5fd0qIpvctvSds6rm/BdQCOwFqoERwF+Bc9Pdr2Gcz0eBWcD2kLb/AG53b98O/Lt7\n+1z3fEuAae7vodB97M/ARYAAvwGuTPe5xTjfKmCWe3s08Kp7Xrl8zgKc5t4uBja6/c7Zcw45968B\nvwT+J9f/tt2+7gPGRbSl7Zzz5UrhQuA1VW1W1ZPAY8D8NPdpyFT1j8DRiOb5wM/d2z8Hrgppf0xV\nu1X1deA14EIRqQJOV9WX1fmLeiTkZzKKqraq6mb39jvATmASuX3Oqqon3LvF7peSw+cMICKTgXnA\nwyHNOX3OMaTtnPMlKEwC3gq5v99tyyXvVdVW93Yb8F73dqxzn+TejmzPaCIyFZiJ88k5p8/ZHUbZ\nChwEfquqOX/OwP3A14H+kLZcP2cFficir4jIQrctbeec95vs5CJVVRHJubQyETkNeBK4VVWPhw6Z\n5uI5q2ofcIGIVABPich5EY/n1DmLyKeAg6r6iohcFu2YXDtn10dUtUVEJgC/FZFdoQ+m+pzz5Uqh\nBTgj5P5kty2XvO1eQuJ+P+i2xzr3Fvd2ZHtGEpFinIDwqKqucptz+pwDVPUYsA74BLl9zpcCnxaR\nfThDvFeIyC/I7XNGVVvc7weBp3CGu9N2zvkSFP4CnCUi00RkBLAAeCbNfUq2Z4Ab3ds3AqtD2heI\nSImITAPOAv7sXpoeF5GL3CyFG0J+JqO4/fsxsFNVfxDyUC6f83j3CgERKQM+Duwih89ZVZeq6mRV\nnYrzf/QFVb2eHD5nERklIqMDt4E5wHbSec7pnnlP1RfwSZyslb3AHenuzzDP5VdAK9CDM3b4JWAs\n0AjsAX4HjAk5/g73vHcTkpEA1Lp/gHuBB3EXM2baF/ARnHHXJmCr+/XJHD/nGmCLe87bgTvd9pw9\n54jzv4xT2Uc5e844GZF/db92BN6b0nnOtqLZGGNMUL4MHxljjPHAgoIxxpggCwrGGGOCLCgYY4wJ\nsqBgjDEmyIKCMXGISJ9bvXK7iPxaREa67ScG+1ljspEFBWPi61TVC1T1POAkcFO6O2SMnywoGOPd\neuB9oQ0icpqINIrIZreW/Xy3/S4RuTXkuLtFZJGIVInIH0OuPman+ByMicsWrxkTh4icUNXTRKQI\np/bSc6r6o4j2keoU6BsHvIxTeuBMYJWqzhKRApyVqRcCnwdKVfVuESl0f/adtJycMVFYlVRj4itz\ny1eDc6Xw44jHBfiuiHwUp9zzJJyyx/tE5IiIzMQpe7xFVY+IyF+An7gF/p5W1a0Yk0EsKBgTX6eq\nXhDn8euA8cDfqWqPW+Gz1H3sYZwrg0rgJ+BskOQGkHnAz0TkB6r6iF+dNyZRNqdgzPCU4+wB0CMi\nl+MMGwU8hVPu+kPAWgARORN4W1X/CydozEpxf42Jy64UjBmeR4EGEdkGbMIpbw2Aqp4UkXXAMXU2\nzAGn+udtItIDnMApcWxMxrCJZmN84k4wbwb+UVX3pLs/xnhhw0fG+EBEzsXZVL3RAoLJJnalYIwx\nJsiuFIwxxgRZUDDGGBNkQcEYY0yQBQVjjDFBFhSMMcYEWVAwxhgT9P8B5FIXkTox720AAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10ac4ba58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(env)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
